#!/usr/bin/python2.4

"""
Squish helps you build, minify, and deploy Gadgets by creating "builds" from
your gadget source, assets, JavaScript, and CSS.

For more infoz, see:
http://code.google.com/p/squish

TODO: unit tests
TODO: automatically inline JS < ~50kB?
TODO: minify inline CSS and JS
TODO: pathname munging is still awkward
"""

from glob import glob
import httplib
import logging
import os
import os.path
import popen2
import re
import sha
import socket
import sys
import urllib
import urllib2
import xml.dom.minidom

import html5lib
import yaml

logging.basicConfig(level=logging.INFO,
                    format="%(asctime)s %(levelname)s %(message)s")


class NoCompressor:
  def __init__(self, fetcher):
    self.fetcher = fetcher

  def FetchAndShrink(self, variant, assets):
    contents = []
    for asset in assets:
      assert asset.startswith("http://") or asset.startswith("data:")
      if asset.startswith("http://"):
        contents += [self.fetcher.Fetch(asset)]
      elif asset.startswith("data:"):
        contents += [asset[5:]]

    return "\n".join(contents)


class YuiCompressor:
  def _run(*args):
    abstract

  def FetchAndShrink(self, variant, assets):
    contents = []
    for asset in assets:
      assert asset.startswith("http://") or asset.startswith("data:"), asset
      if asset.startswith("http://"):
        contents += [self.fetcher.Fetch(asset)]
      elif asset.startswith("data:"):
        contents += [asset[5:]]

    big = "".join(contents)
    small = self._run(big, variant)

    logging.info("%s: was %d bytes in %d includes, now %d bytes "
                 "-- %d%% of original.",
                 variant, len(big), len(assets), len(small),
                 100*float(len(small))/max(1, len(big)))

    return small


class LocalYuiCompressor(YuiCompressor):
  def __init__(self, yui_compressor_jar, fetcher):
    self.yui_compressor_jar = yui_compressor_jar
    self.fetcher = fetcher

  def _run(self, input, variant):
    assert variant in ("js", "css")

    yui_compressor = "java -jar %s --type %s"
    r, w = popen2.popen2(yui_compressor % (self.yui_compressor_jar, variant))
    w.write(input)
    w.write("\n")
    w.close()
    return r.read()


class RemoteYuiCompressor(YuiCompressor):
  def __init__(self, yui_compressor_server, fetcher):
    self.yui_compressor_server = yui_compressor_server
    self.fetcher = fetcher
    self.conn = None

  def _run(self, input, variant):
    assert variant in ("js", "css")
    path = "/%s" % variant

    params = urllib.urlencode({'data': input})
    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
    if not self.conn:
      logging.info("Connecting to %s" % self.yui_compressor_server)
      self.conn = httplib.HTTPConnection(self.yui_compressor_server)
    self.conn.request('POST', path, params, headers)
    response = self.conn.getresponse()
    assert response.status == 200, "received non-200 status from reducisaurus"
    data = response.read()
    return data


def input_url_to_input_disk_path(input_path_root, input_path_root_url, asset):
  assert asset.startswith(input_path_root_url)
  asset = asset.replace(input_path_root_url, input_path_root, 1)
  return asset


class Fetcher:
  def __init__(self, input_path_root, input_path_root_url):
    self.cache = {}
    self.input_path_root = input_path_root
    self.input_path_root_url = input_path_root_url

  def Fetch(self, url):
    if url.startswith(self.input_path_root_url):
      local_path = input_url_to_input_disk_path(self.input_path_root,
                                                self.input_path_root_url, url)
      if not os.path.exists(local_path):
        logging.error("unable to fetch %s", local_path)
        sys.exit(1)
      big = open(local_path, "r").read()
    else:
      if url in self.cache:
        big = self.cache[url]
      else:
        logging.info("fetching external URL %s...", url)
        big = urllib2.urlopen(url).read()
        self.cache[url] = big

    return big


class Gadget:
  """Represents the Gadget"""
  def __init__(self, gadget_spec_fn):
    self.gadget_dom = xml.dom.minidom.parse(gadget_spec_fn)

    content_node = self.gadget_dom.getElementsByTagName("Content")[0]
    content_html = "".join([x.data for x in content_node.childNodes])

    self.content_dom = html5lib.HTMLParser(strict=False,
        tree=html5lib.treebuilders.getTreeBuilder("dom")).parse(content_html)

  def TweakModulePrefsAttributes(self, visitor):
    module_prefs = self.gadget_dom.getElementsByTagName("ModulePrefs")[0]
    for attr_name, attr_value in module_prefs.attributes.items():
      new_value = visitor.VisitModulePrefsAttribute(attr_name, attr_value)
      if new_value:
        module_prefs.setAttribute(attr_name, new_value)

  def TweakLocaleAttributes(self, visitor):
    locale_tags = self.gadget_dom.getElementsByTagName("Locale")
    for tag in locale_tags:
      for attr_name, attr_value in tag.attributes.items():
        new_value = visitor.VisitModulePrefsAttribute(attr_name, attr_value)
        if new_value:
          tag.setAttribute(attr_name, new_value)

  def TweakContentDom(self, visitor):
    visitor.VisitContentDom(self.content_dom)

  def TweakGadgetDom(self, visitor):
    visitor.VisitGadgetDom(self.gadget_dom)

  def GetNewGadgetSpec(self):
    """Builds a new gadget spec as a string"""
    content_head_node = self.content_dom.getElementsByTagName("head")[0]
    content_body_node = self.content_dom.getElementsByTagName("body")[0]

    nodes_we_like = [x.toxml() for x in content_head_node.childNodes
                     if x.nodeName != "#text"]
    nodes_we_like += [x.toxml() for x in content_body_node.childNodes]
    new_content_body_html = "".join(nodes_we_like)

    # strip the existing <Content> section from <Module> and replace it with
    # our new Content section.
    module_tag = self.gadget_dom.getElementsByTagName("Module")[0]
    module_tag.removeChild(module_tag.getElementsByTagName("Content")[0])

    content_tag = self.gadget_dom.createElement("Content")
    content_tag.setAttribute("type", "html")
    content_tag.appendChild(self.gadget_dom.createCDATASection(
        new_content_body_html))
    module_tag.appendChild(content_tag)

    return self.gadget_dom.toxml()


class BaseMinifier:
  """Framework for implementing minifiers of external assets."""
  def __init__(self, crunch_monster, writer, gadget_name):
    self.urls = []
    self.crunch_monster = crunch_monster
    self.writer = writer
    self.gadget_name = gadget_name

  def VisitContentDom(self, dom):
    content_head_node = dom.getElementsByTagName("head")[0]
    content_body_node = dom.getElementsByTagName("body")[0]
    for suspect_list in (content_head_node, content_body_node):
      for node in suspect_list.childNodes:
        if self._Matches(node):
          self._Handle(node)
          if self._ShouldRemoveNode(node):
            suspect_list.removeChild(node)
    self._Minify(dom)

  def _Minify(self, dom):
    if self.urls:
      compiled_output = self.crunch_monster.FetchAndShrink(self.variant,
                                                           self.urls)
      url = self.writer.FromStringWithFingerprint(
          self.gadget_name + ".min.%s" % self.variant,
          compiled_output)
      self._UpdateDom(dom, url)


class CssMinifier(BaseMinifier):
  """Minifies <link rel="stylesheet"...> CSS."""

  variant = "css"

  def _Matches(self, node):
    return (node.nodeName == "link"
            and node.attributes["rel"].value == "stylesheet")


  def _Handle(self, node):
    href = node.attributes["href"].value
    self.urls.append(href)


  def _ShouldRemoveNode(self, node):
    return True


  def _UpdateDom(self, dom, url):
    new_css_tag = dom.createElement("link")
    new_css_tag.setAttribute("type", "text/css")
    new_css_tag.setAttribute("rel", "stylesheet")
    new_css_tag.setAttribute("href", url)

    # CSS should always be before any SCRIPT tags
    content_head_node = dom.getElementsByTagName("head")[0]
    non_rewritten_script_tags = dom.getElementsByTagName("script")

    if non_rewritten_script_tags.length:
      content_head_node.insertBefore(new_css_tag, non_rewritten_script_tags[0])
    else:
      content_head_node.appendChild(new_css_tag)


class JsMinifier(BaseMinifier):
  """Minifies <script src=> JavaScript."""

  variant = "js"

  def _Matches(self, node):
    return node.nodeName == "script"

  def _Handle(self, node):
    if "src" in node.attributes.keys():
      src = node.attributes["src"].value
      if self._IsEligibleSrc(src):
        self.urls.append(src)
    else:
      inline_src = self._GetInlineSrc(node)
      if self._IsEligibleInlineSrc(inline_src):
        self.urls.append("data:" + inline_src)

  def _GetInlineSrc(self, node):
    return "".join([x.data for x in node.childNodes])

  def _ShouldRemoveNode(self, node):
    # We only remove the node if we're going to rewrite it.
    has_src_attribute = "src" in node.attributes.keys()

    if has_src_attribute and self._IsEligibleSrc(node.attributes["src"].value):
      return True

    if not has_src_attribute and self._IsEligibleInlineSrc(self._GetInlineSrc(node)):
      return True

    return False

  def _IsEligibleSrc(self, src):
    return not "jquery" in src

  def _IsEligibleInlineSrc(self, inline_src):
    matcher = re.compile("__[a-zA-Z_]+__")
    return not matcher.search(inline_src)

  def _UpdateDom(self, dom, url):
    new_script_tag = dom.createElement("script")
    new_script_tag.setAttribute("src", url)
    dom.getElementsByTagName("head")[0].appendChild(new_script_tag)


class RemoveComments:
  def VisitGadgetDom(self, dom):
    if not dom.hasChildNodes():
      return

    for x in dom.childNodes:
      if x.nodeType == 8:  # comment
        dom.removeChild(x)
      else:
        self.VisitGadgetDom(x)


class FixScriptTags:
  """Ensures each <script> tag has a matching </script>."""
  def VisitContentDom(self, dom):
    for x in dom.getElementsByTagName("script"):
      if not x.childNodes:
        x.appendChild(dom.createTextNode(""))


class Proxifier:
  """Generic class that rewrites attribute values to use the gmodules proxy."""
  hostname_prefix = "www"

  def __init__(self, writer):
    self.writer = writer

  def VisitContentDom(self, dom):
    for tag in dom.getElementsByTagName(self.tag_name):
      if tag.hasAttribute(self.attr_name):
        url = tag.getAttribute(self.attr_name)

        if self.writer.IsUrlEligibleForOutput(url):
          url = self.writer.CopyInputWithFingerprint(url)

        url = "http://%s.gmodules.com/ig/proxy?url=%s" % (self.hostname_prefix,
                                                          urllib.quote_plus(url))
        tag.setAttribute(self.attr_name, url)


class ImageProxifier(Proxifier):
  """Rewrites <img> tags."""
  tag_name = "img"
  attr_name = "src"


class JsProxifier(Proxifier):
  """Rewrites <img> tags."""
  tag_name = "script"
  attr_name = "src"


class CssProxifier(Proxifier):
  """Rewrites <link> href attributes."""
  tag_name = "link"
  attr_name = "href"


class ThumbnailScreenshotRewriter:
  """Replaces URLs to thumbnail= and screenshot= values that are in our input
  directory with URLs to versions in our output directory.
  """
  def __init__(self, writer):
    self.writer = writer

  def VisitModulePrefsAttribute(self, attr_name, attr_value):
    if attr_name not in ("screenshot", "thumbnail"):
      return

    if self.writer.IsUrlEligibleForOutput(attr_value):
      return self.writer.CopyInputWithFingerprint(attr_value)


class MessageBundleRewriter:
  """Replaces URLs to messages= with URLs to our versions in the output
  directory."""

  def __init__(self, writer):
    self.writer = writer

  def VisitModulePrefsAttribute(self, attr_name, attr_value):
    # TODO: bug?
    if attr_name not in ("messages"):
      return

    if self.writer.IsUrlEligibleForOutput(attr_value):
      return self.writer.CopyInputWithFingerprint(attr_value)


class Writer:
  def __init__(self, input_path_root, input_path_root_url, output_path_root,
               output_path_root_url, gadget_name):
    assert not output_path_root_url.startswith(input_path_root_url)
    assert not input_path_root_url.startswith(output_path_root_url)

    self.output = {}

    self.input_path_root = input_path_root
    self.input_path_root_url = input_path_root_url
    self.output_path_root = output_path_root
    self.output_path_root_url = output_path_root_url

    self.gadget_name = gadget_name

  def _InputUrlToOutputUrl(self, input_url):
    assert input_url.startswith(self.input_path_root_url)
    output_url = input_url.replace(self.input_path_root_url,
                                   self.output_path_root_url, 1)
    assert output_url.startswith(self.output_path_root_url)
    return output_url

  def _Fingerprint(self, s):
    return sha.sha(s).hexdigest()[:5]

  def GetOutputFiles(self):
    return self.output

  def IsUrlEligibleForOutput(self, input_url):
    return input_url.startswith(self.input_path_root_url)

  def _FingerprintPath(self, filename, fingerprint):
    root, extension = os.path.splitext(filename)
    root += ".%s" % fingerprint
    filename = root + extension
    return filename

  def CopyInputWithFingerprint(self, input_url):
    assert self.IsUrlEligibleForOutput(input_url)

    source_on_disk = input_url_to_input_disk_path(self.input_path_root,
                                                  self.input_path_root_url,
                                                  input_url)

    contents = open(source_on_disk, "r").read()
    fp = self._Fingerprint(contents)

    source_on_disk = source_on_disk.replace(self.input_path_root, "", 1)
    source_on_disk = source_on_disk.lstrip("/")

    target_on_disk = self._FingerprintPath(source_on_disk, fp)

    output_url = self._InputUrlToOutputUrl(input_url)
    output_url = self._FingerprintPath(output_url, fp)

    self.output[target_on_disk] = contents

    return output_url

  def FromString(self, filename, body):
    self.output[filename] = body

  def FromStringWithFingerprint(self, filename, body):
    fp = self._Fingerprint(body)
    filename = self._FingerprintPath(filename, fp)
    self.output[filename] = body

    return os.path.join(self.output_path_root_url, filename)

  def Write(self):
    """Writes the output to disk."""

    for output_filename, output_contents in self.output.iteritems():
      # all these paths should be relative
      assert not output_filename.startswith("/")

      output_filename = os.path.join(self.output_path_root, output_filename)
      directory = os.path.dirname(output_filename)
      if not os.path.exists(directory):
        logging.info("creating directory: %s", directory)
        os.makedirs(directory)

      out = open(output_filename, "w")
      out.write(output_contents)
      out.close()

      logging.info("wrote %s: %d bytes", output_filename, len(output_contents))


def Squish(gadget, fetcher, cruncher, writer, gadget_name):
  module_pref_attribute_tweakers = [
      ThumbnailScreenshotRewriter(writer),
  ]

  for tweaker in module_pref_attribute_tweakers:
    gadget.TweakModulePrefsAttributes(tweaker)

  localization_tweakers = [
      MessageBundleRewriter(writer),
  ]

  for tweaker in localization_tweakers:
    gadget.TweakLocaleAttributes(tweaker)

  dom_tweakers = [
      CssMinifier(cruncher, writer, gadget_name),
      JsMinifier(cruncher, writer, gadget_name),
      ImageProxifier(writer),
      CssProxifier(writer),
      JsProxifier(writer),
      FixScriptTags(),
  ]

  for tweaker in dom_tweakers:
    gadget.TweakContentDom(tweaker)

  gadget_tweakers = [
      RemoveComments(),
  ]

  for tweaker in gadget_tweakers:
    gadget.TweakGadgetDom(tweaker)

  writer.FromString(gadget_name + ".xml", gadget.GetNewGadgetSpec())

  return writer.GetOutputFiles()


def _MaybeMakePathAbsolute(o, attr, trailing_slash=True):
  if not o.get(attr, ""):
    return None

  if not o[attr].startswith("/"):
    o[attr] = os.path.abspath(o[attr])

  if trailing_slash:
    if not o[attr].endswith("/"):
      o[attr] += "/"


def _ValidateRequiredUrl(config, attr, errors):
  if (not config[attr]
      or not config[attr].startswith("http://")
      or not config[attr].endswith("/")):
    errors.append("--%s must start with http:// and end with /." % attr)


def _ValidatePathExists(config, attr, errors):
  if not os.path.exists(config[attr]):
    errors.append("--%s must exist" % attr)


def _ValidateBooleanWithDefault(config, attr, default_value, _):
  if not attr in config:
    config[attr] = default_value


def _ValidateHostnameExists(config, attr, errors):
  if attr not in config:
    errors.append("--%s is probably missing" % attr)
    return

  hostname = config[attr]
  try:
    socket.gethostbyname(hostname)
  except socket.gaierror, e:
    errors.append("Invalid hostname specified for --%s: %s: %s"
                  % (attr, hostname, e.message))


def _ValidateOneOf(config, attrs, errors):
  keys_present = [True for x in attrs if x in config]
  if not keys_present:
    errors.append("At least one of the following options must be present: %s"
                  % ", ".join(attrs))
  elif len(keys_present) == 2:
    errors.append("Only one of the following options may be present: %s"
                  % ", ".join(attrs))


def ValidateOptions(config):
  errors = []
  _MaybeMakePathAbsolute(config, "input_path_root")
  _MaybeMakePathAbsolute(config, "output_path_root")
  _MaybeMakePathAbsolute(config, "yui_compressor_jar", trailing_slash=False)

  _ValidatePathExists(config, "input_path_root", errors)
  _ValidateRequiredUrl(config, "input_path_root_url", errors)

  _ValidatePathExists(config, "output_path_root", errors)
  _ValidateRequiredUrl(config, "output_path_root_url", errors)
  _ValidateOneOf(config, ("yui_compressor_jar", "yui_compressor_server"),
                 errors)

  if not "yui_compressor_jar" in config:
    _ValidateHostnameExists(config, "yui_compressor_server", errors)
  else:
    _ValidatePathExists(config, "yui_compressor_jar", errors)

  _ValidateBooleanWithDefault(config, "minify", True, errors)

  return errors


def main(args):
  options = yaml.load(open(args.pop()).read())
  errors = ValidateOptions(options)

  if errors:
    for x in errors:
      logging.error("bad args: %s", x)
    sys.exit(1)
  else:
    input_path_root = options["input_path_root"]
    input_path_root_url = options["input_path_root_url"]
    output_path_root = options["output_path_root"]
    output_path_root_url = options["output_path_root_url"]
    yui_compressor_jar = options.get("yui_compressor_jar", "")
    minify = options.get("minify", True)
    input_specs = options["input_specs"]
    exclude_specs = options.get("exclude_specs", [])
    yui_compressor_server = options.get("yui_compressor_server", "")

    excluded = []
    for exclude_pattern in exclude_specs:
      excluded += glob(exclude_pattern)

    for pattern in input_specs:
      logging.info("processing pattern: %s", pattern)
      for gadget_spec in [x for x in glob(pattern) if x not in excluded]:
        logging.info("processing gadget spec: %s", gadget_spec)

        gadget_spec = os.path.abspath(gadget_spec)
        gadget_name = gadget_spec.replace(input_path_root, "", 1)
        gadget_name, _ = os.path.splitext(gadget_name)
        logging.info("gadget_name = %s", gadget_name)

        gadget = Gadget(gadget_spec)
        fetcher = Fetcher(input_path_root, input_path_root_url)
        if minify:
          if yui_compressor_server:
            cruncher = RemoteYuiCompressor(yui_compressor_server, fetcher)
          else:
            cruncher = LocalYuiCompressor(yui_compressor_jar, fetcher)
        else:
          cruncher = NoCompressor(fetcher)
        writer = Writer(input_path_root, input_path_root_url,
                        output_path_root,
                        output_path_root_url,
                        gadget_name)

        Squish(gadget, fetcher, cruncher, writer, gadget_name)

        writer.Write()


if __name__ == "__main__":
  if "DOCTEST" in os.environ:
    import doctest
    doctest.testmod()
    sys.exit(0)

  main(sys.argv)
